package archer.back.controllers

import java.io.ByteArrayInputStream
import java.net.URL
import java.util.{ Date, UUID }

import akka.util.ByteString
import archer.back.models._
import utils.EncoderEx

import scala.concurrent.{ ExecutionContext, Future }
import cats.implicits._
import org.jsoup.Jsoup
import org.jsoup.nodes.{ Document, Element }
import org.slf4j.LoggerFactory
import org.xarcher.emiya.utils.{ FutureLimited, FutureTimeLimited }
import play.api.libs.ws.WSRequest
import play.api.http.HeaderNames
import play.api.libs.ws.WSClient

import scala.annotation.tailrec
import scala.collection.JavaConverters._
import scala.util.Try

import models.CrawlTables._

class CrawlUrlAction(
  url: String,
  referer: String,
  plan: CrawlPlanRow,
  action: CrawlActionRow,
  futureTimeLimited: FutureTimeLimited,
  futureLimited: FutureLimited,
  wsClient: WSClient,
  currentDeep: Int)(implicit ec: ExecutionContext) {

  val logger = LoggerFactory.getLogger(classOf[CrawlUrlAction])

  lazy val URLOpt = Try { new URL(url) }

  lazy val hostOpt =
    URLOpt.flatMap(s => Try { s.getHost }).toOption.filterNot(_.isEmpty)

  lazy val protocolOpt =
    URLOpt.flatMap(s => Try { s.getProtocol }).toOption.filterNot(_.isEmpty)

  //频率控制
  def limitFuture[T](f: () => Future[T]): Future[T] =
    futureLimited.limit(
      () => futureTimeLimited.limit(() => f(), 1, s"url:$url"),
      1,
      s"url:$url")

  lazy val currentCache =
    CacheWrap(planId = plan.id, actionId = action.id, cacheUrl = url)

  //本次抓取任务主键
  //lazy val esId = currentCache.urlKey

  case class ResWrap(response: WSRequest#Response, endDate: Date)

  lazy val startTime = new Date()
  lazy val wsAction: Future[ResWrap] = {
    logger.info(
      s"正在抓取链接：${url}，链接 ElasticSearch id 为：${currentCache.elasticKey}")
    limitFuture(() => {
      startTime
      wsClient.url(url).withHttpHeaders(HeaderNames.REFERER -> referer).get
    }).map(s => ResWrap(s, new Date()))
  }

  @tailrec
  private def getPathGen(dom: Element, paths: List[Int]): List[Int] = {
    Option(dom.parent()) match {
      case Some(s) =>
        s.children().asScala.toList.zipWithIndex.find(s => s._1 == dom) match {
          case Some((_, index)) =>
            getPathGen(s, index :: paths)
          case _ =>
            throw new Exception("找不到本节点的 index")
        }
      case _ =>
        paths
    }
  }

  def getPath(dom: Element): List[Int] = {
    getPathGen(dom, List.empty)
  }

  case class ParsedBody(
    cssBodyOpt: Option[Document],
    lawParsedBody: Option[String],
    subLinks: List[SubUrl])
  object ParsedBody {
    val empty: ParsedBody = ParsedBody(
      cssBodyOpt = Option.empty,
      lawParsedBody = Option.empty,
      subLinks = List.empty)
  }

  def jsoupBodyToResult(bodyBytes: ByteString): Try[ParsedBody] = {
    Try {
      val doc =
        Jsoup.parse(new ByteArrayInputStream(bodyBytes.toArray), null, url)
      val charset = doc.charset

      val cssPrefix = "lolita-gentleman-"

      val hrefLinks = doc
        .select("*[href]")
        .asScala
        .toList
        .map { dom =>
          val urlOpt = Option(dom.attr("abs:href"))
            .map(_.trim)
            .filterNot(_.isEmpty)
          urlOpt.map { s =>
            val autalUrl =
              EncoderEx.encodeURI(s, charset).getOrElse(s)
            val newClass = cssPrefix + UUID.randomUUID().toString
            dom.attr("href", autalUrl)
            dom.addClass(newClass)
            (autalUrl, getPath(dom), newClass)
          }
        }
        .collect { case Some(s) => SubUrl(s._1, s._2, s._3) }

      val srcLinks = doc
        .select("*[src]")
        .asScala
        .toList
        .map { dom =>
          val urlOpt = Option(dom.attr("abs:src"))
            .map(_.trim)
            .filterNot(_.isEmpty)
          urlOpt.map { s =>
            val autalUrl =
              EncoderEx.encodeURI(s, charset).getOrElse(s)
            val newClass = cssPrefix + UUID.randomUUID().toString
            dom.attr("src", autalUrl)
            dom.addClass(newClass)
            (autalUrl, getPath(dom), newClass)
          }
        }
        .collect { case Some(s) => SubUrl(s._1, s._2, s._3) }

      ParsedBody(
        cssBodyOpt = Option(doc),
        lawParsedBody =
          Option(bodyBytes.decodeString(doc.outputSettings.charset)),
        subLinks = (hrefLinks ::: srcLinks))
    }

  }

  //不需要考虑失败的情况，因为失败后不会用到这个判断
  lazy val needFetchChildren: Future[Boolean] = wsAction.map { responseWrap =>
    (for {
      host <- hostOpt
      protocol <- protocolOpt
      planHost <- Try {
        new URL(plan.rootUrl).getHost
      }.toOption.filterNot(_.isEmpty)
    } yield {
      (protocol
        .startsWith("http")) && (host === planHost) && (responseWrap.response.contentType
          .startsWith("text/html"))
    }).getOrElse(false)
  }

  def failedResult(throwable: Throwable): IndexTaskWrap = {
    logger.info(s"抓取url：${url} 发生错误", throwable)

    val failedTime = new Date()

    val common = CommonBody(
      url = url,
      contentType = Option.empty,
      body = Option.empty,
      law_body = Map.empty,
      textBody = Option.empty,
      referer = Option(referer),
      law_text_body = Map.empty,
      cssBody = Option.empty,
      host = hostOpt,
      protocol = protocolOpt,
      deep = currentDeep,
      wasteTime = failedTime.getTime - startTime.getTime,
      requestTime = startTime,
      status = 1001,
      contentLength = -1,
      subs = List.empty,
      //needFetchedChildern = false,
      //fetchedChildern = false,
      planId = plan.id,
      actionId = action.id)

    IndexTaskWrap(
      cache = currentCache,
      body = common,
      afterIndexed = { () =>
        logger.info(
          s"尝试链接发生网络错误，但仍然创建了 Id 为：${currentCache.elasticKey}，url 为：${currentCache.cacheUrl} 的链接信息")
      })

  }

  def successResult(parsedBody: ParsedBody, resWrap: ResWrap): IndexTaskWrap = {
    val indexTextOpt = parsedBody.lawParsedBody
    val indexLawTextOpt = parsedBody.cssBodyOpt.map(_.text())
    val cssBodyOptStr = parsedBody.cssBodyOpt.map(_.outerHtml)

    val extMap = indexTextOpt match {
      case Some(text) =>
        text
          .grouped(8000)
          .toList
          .foldLeft((List.empty[(String, String)], "", 0)) {
            case ((list, str, index), each) =>
              (((("law_body_" + index) -> (str + each)) :: list),
                each.takeRight(200),
                index + 1)
          }
          ._1
      case _ =>
        List.empty
    }

    val lawTextExtMap = indexLawTextOpt match {
      case Some(text) =>
        text
          .grouped(8000)
          .toList
          .foldLeft((List.empty[(String, String)], "", 0)) {
            case ((list, str, index), each) =>
              (((("law_text_body_" + index) -> (str + each)) :: list),
                each.takeRight(200),
                index + 1)
          }
          ._1
      case _ =>
        List.empty
    }

    val common = CommonBody(
      url = url,
      contentType = Option(resWrap.response.contentType),
      body = indexTextOpt,
      textBody = indexLawTextOpt,
      referer = Option(referer),
      law_body = extMap.toMap,
      law_text_body = lawTextExtMap.toMap,
      cssBody = cssBodyOptStr,
      host = hostOpt,
      protocol = protocolOpt,
      deep = currentDeep,
      wasteTime = resWrap.endDate.getTime - startTime.getTime,
      requestTime = new Date(),
      contentLength = resWrap.response.bodyAsBytes.size,
      status = resWrap.response.status,
      subs = parsedBody.subLinks,
      planId = plan.id,
      actionId = action.id)

    IndexTaskWrap(cache = currentCache, body = common, afterIndexed = { () =>
      logger.info(
        s"创建了 Id 为:${currentCache.elasticKey}，url 为：${currentCache.cacheUrl} 的链接信息")
    })
  }

  def indexUrl: Future[IndexTaskWrap] = {
    wsAction
      .flatMap { actionWrap =>
        needFetchChildren.map { need =>
          val parsedBodyModel =
            if (need)
              jsoupBodyToResult(actionWrap.response.bodyAsBytes)
                .getOrElse(ParsedBody.empty)
            else
              ParsedBody.empty

          successResult(parsedBodyModel, actionWrap)
        }
      }
      .recover {
        case e: Exception =>
          failedResult(e)
      }
  }

}
